#coding=utf-8
import sys,os
import shutil
import codecs

reload(sys)
sys.setdefaultencoding('utf-8')

def InitVocabDict(vocabFile):
    #words = []
    words = dict()
    fp = codecs.open(vocabFile, 'r', encoding='UTF-8')
    lines = fp.readlines()
    for line in lines:
        line = line.strip()
        #words.append(line)
        words[line] = ' '
 
    fp.close()
 
    return words

oral_words = ['呢','啊','啦','嗯','哼','呵','呀','咯','呃','嘿','哦','呗']

# step=1, 处理AA重复的现象，step=2, 处理ABAB重复的现象等等
def deal_oral(src_file, dst_file, step):
    wordsCollect = InitVocabDict('./vocab_20180313')

    src = codecs.open(src_file, 'r', encoding='UTF-8')
    dst = codecs.open(dst_file, 'w', encoding='UTF-8')

    src_lines = src.readlines()
    new_line = ''
    last_pattern = ''

    for line in src_lines:
        new_line = ''
        repeat_num = 0
        repeat_char = ''
        line_len = len(line)

        for b in range(0,step): #为防止ABAB被取成CABA的形式，设置不同的起始点

            if b > 0: #当前在上次结果的基础上进行处理
                line = new_line

            new_line = line[0:b]
                
            for i in range(b,line_len,step):
                pattern = line[i:i+step]

                if (step == 1) and (pattern in oral_words):
                    continue #处理语气词

                else: #非语气词
                    if pattern == last_pattern and not (pattern.encode('utf-8').isalpha() or pattern.isdigit()): #重复
                        repeat_num += 1
                        if repeat_num == 1:
                            repeat_pattern = last_pattern + pattern
                        else:
                            repeat_pattern += pattern

                    # 重复停止时
                    else:
                        if ((step == 1) and (repeat_num>0) and (repeat_num<=2)) or ((step == 2) and (repeat_num == 1)): #AA, AAA, ABAB形式可能是合理的，查词典
                            if wordsCollect.get(repeat_pattern): #如果重复的词可以在词典中查到
                                new_line += repeat_pattern[step:]
                                new_line += pattern
                            else:
                                new_line += pattern

                        else: #无重复的情况
                            new_line += pattern
                            
                        repeat_num = 0
                        repeat_pattern = ''

                    last_pattern = pattern
                                                            
        dst.write(new_line)

    src.close()
    dst.close()


if __name__ == "__main__":

    src_file = sys.argv[1]
    dst_dir = sys.argv[2]

    #src_file = 'D:/data/test_speech/5-th_meeting/with_oral.txt'
    #dst_dir = 'D:/data/test_speech/5-th_meeting/oral_dealed'

    max_pattern_len = 4

    for i in range(max_pattern_len,0,-1):
        if i == max_pattern_len:
            temp_file = src_file
        
        dst_file = dst_dir + '_' + str(i)

        deal_oral(temp_file, dst_file, i)
        
        if i != max_pattern_len:
            os.remove(temp_file)

        temp_file = dst_file


    if os.path.isfile(temp_file):
        shutil.copyfile(temp_file, dst_dir)
        os.remove(temp_file)
